package analisador;

import java.io.*;
import java.net.URLDecoder;
import java.sql.*;
import java.util.Scanner;
import java.util.StringTokenizer;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.demo.html.HTMLParser;
import org.apache.lucene.util.Version;

public class Projeto {

    static Connection con;

    public static void main(String[] args) throws Exception {
        con = DriverManager.getConnection(
                "jdbc:mysql://localhost:3306/hadoop",
                "root",
                "root");

        System.out.println("Memória Máxima:" + Runtime.getRuntime().maxMemory());
        System.gc();

        geraFileTokens();
        carrega();


    }

    public static void geraComentarioDecodificado() throws Exception {
        Statement stmt = con.createStatement();

        ResultSet rs = stmt.executeQuery("SELECT id,comentario FROM comentario LIMIT 800000, 100000");
        System.out.println("Atualizando...");
        File arquivo = new File("updates6.txt");
        FileWriter fileWriter = new FileWriter(arquivo, false);
        PrintWriter printWriter = new PrintWriter(arquivo);
        long cont = 0;

        while (rs.next()) {
            long id = rs.getLong("id");
            String comentario = rs.getString("comentario");
            String comentarioDecode = URLDecoder.decode(comentario, "UTF-8").trim().replaceAll("[\n|\r|\t|,|\u2028|\u2029|\u2229]", " ").replaceAll("  +", " ");
            String update = String.format("%d,%s", id, comentarioDecode);
            printWriter.println(update);
            if (cont++ == 100) {
                printWriter.flush();
                cont = 0;
            }
        }
        printWriter.flush();
        printWriter.close();
    }

    public static void carrega() throws Exception {
        File arquivo = new File("tokensHadoop.txt");

        long cont = 0;
        String[] linha = null;
        try {
            Scanner leitor = new Scanner(arquivo);
            while (leitor.hasNextLine()) {
                linha = leitor.nextLine().split(",");
                if (linha.length > 1) {
                    PreparedStatement p = con.prepareStatement("UPDATE comentario SET TOKENS=? WHERE ID=?");
                    p.setString(1, linha[1]);
                    p.setInt(2, Integer.parseInt(linha[0]));
                    p.execute();
                    p.close();
                }
                if (cont++ % 1000 == 0) {
                    System.out.println(cont);
                }
            }
        } catch (FileNotFoundException ex) {

            System.err.println("Arquivo não encontrado");
        } catch (Exception ex) {
            System.err.println(linha[0]);
            System.err.println(ex.getMessage());
        }
    }

    private static void geraFileTokens() {
        long id = 0;
        String str = null;
        try {
            File arquivo = new File("tokensHadoop.txt");
            FileWriter fileWriter = new FileWriter(arquivo, false);
            PrintWriter printWriter = new PrintWriter(arquivo);

            String q = "SELECT ID, AUTOR, COMENTARIO FROM comentario where id=314";
            ResultSet rs = con.createStatement().executeQuery(q);
            Analyzer an = new CustomAnalyzer(Version.LUCENE_35);

            long cont = 0;
 
            while (rs.next()) {

                if (rs.getString("AUTOR").equals("hudson") || rs.getString("AUTOR").equals("hadoop")) {
                    continue;
                }

                id = rs.getLong("ID");
                printWriter.print(id + ", ");
                String comentario = rs.getString("COMENTARIO");

                HTMLParser htmlParser = new HTMLParser(new StringReader(comentario));


                str = convertReaderToString(htmlParser.getReader());
System.out.println(str);
                str = str.replaceAll("[\n|\r|\t|,|\u2028|\u2029|\u2229|\u2122]", " ").replaceAll("[\\d+?]", "");
                str = str.replace("..","");
                int start;
                do {
                    start = str.indexOf("http://");
                    if (start != -1) {
                        int end = str.indexOf(" ", start);
                        if (end == -1) {
                            end = str.length();
                        }
                        printWriter.print(str.substring(start, end) + " ");
                        str = str.replace(str.substring(start, end), "");
                    }
                } while (start != -1);
                
                
                do {
                    start = str.indexOf("https://");
                    if (start != -1) {
                        int end = str.indexOf(" ", start);
                        if (end == -1) {
                            end = str.length();
                        }
                        printWriter.print(str.substring(start, end) + " ");
                        str = str.replace(str.substring(start, end), "");
                    }
                } while (start != -1);

        
                StringTokenizer tokenizer = new StringTokenizer(str);
                while (tokenizer.hasMoreTokens()) {
                    String token = tokenizer.nextToken();
                    
                    int pos = token.indexOf(".");
                    if(pos!=-1 && pos != token.length()-1){
                        str = str.replace(token, " ");
                         printWriter.print(token + " ");
                    }
                }
                System.out.println(str);
                TokenStream stream = an.tokenStream("comentario", new StringReader(str));

                TermAttribute term = stream.addAttribute(TermAttribute.class);
                while (stream.incrementToken()) {
                    printWriter.print(term.term() + " ");
                }
                printWriter.println();

                if (cont++ % 10000 == 0) {
                    System.out.println(cont + ":" + Runtime.getRuntime().freeMemory());
                }
            }
            printWriter.close();
            fileWriter.close();

        } catch (FileNotFoundException ex) {
            System.err.println("Arquivo não encontrado");
        } catch (Exception ex) {
            System.err.println("id: " + id);
            System.err.println(str);
            System.err.println(ex.getMessage());
        }
    }

    public static String convertReaderToString(Reader reader)
            throws IOException {

        if (reader == null) {
            return "";
        }
        Writer writer = new StringWriter();

        char[] buffer = new char[1024];

        int n;

        while ((n = reader.read(buffer)) != -1) {
            writer.write(buffer, 0, n);
        }

        return writer.toString();
    }
}
